
install.packages(...) # Always keep installation commands commented out


install.packages('devtools', ...)

# remove.packages("devtools") # Uninstall any CRAN package


library(...) 

# or 

#library('devtools')


if (!requireNamespace("BiocManager", quietly = TRUE)) 
    install.packages("BiocManager") # this piece of code checks if BiocManager is installed. 
# If is not installed, it will do it for you. It does nothing if BiocManager is already installed.

# BiocManager::install("GenomicRanges")

#or 

#BiocManager::install(c("GenomicRanges", "ConnectivityMap"))


devtools...("tidyverse/googlesheets4")


# Load googlesheets4 now from the library
library(...)


#--------- Install packages to for today's session ----------#
#install.packages("tidyverse", dependencies = TRUE) # This package should already be installed on Jupyter Hub

install.packages("writexl", dependencies = TRUE) # This package is NOT already installed on Jupyter Hub

#--------- Load packages to for today's session ----------#
library(tidyverse)

# readxl, used for reading xlsx files, is not a core component of the tidyverse
library(readxl) 
library(writexl)


# ?read_csv

# Import our microbe.csv file from the data folder
microbes <- read_csv(file = "...",
                     col_names = TRUE,
                     col_types = cols()  # Producing a blank cols() specification suppresses the normal parsing output
                    )

# Check out the structure of our table
str(microbes)


# read_csv() doesn't work for excel files
head(read_csv("..."))


# The readxl package is not a core component of the tidyverse so we need to load it
library(readxl) # Note that we've already loaded it in section 1.5.0

# let's take a peek at the miscellaneous.xlsx sheet
head(...("data/miscellaneous.xlsx"))


# grab the excel sheet names 
...("data/miscellaneous.xlsx")


# read in a specific sheet and range with read_excel()
read_excel(path = "data/miscellaneous.xlsx", 
           sheet = ..., range = "...")


# read in an excel files by a specific row range

read_excel("data/miscellaneous.xlsx", sheet = "microbes", range = ...)


# read in an excel files by a specific column range

head(read_excel("data/miscellaneous.xlsx", 
           sheet = "microbes", range = ...))


# Use lapply and provide a list of excel sheet names, then apply a function to each element (Sheet name) of the list!
excel_sheets_list <- lapply(X = ...("data/miscellaneous.xlsx"), # this will set X to a character vector
                            FUN = ...,
                            path = "..." # This is an argument for read_excel()
                           ) 

# What is the structure of our sheets_list?
str(excel_sheets_list)


# You can see the structure of our first list element. 
# Remember the difference between [[]] and []?
str(excel_sheets_list...)


# Pull a single column from our tibble
print("Indexing a column from a tibble is still a tibble")
str(excel_sheets_list[[1]]...)

# Cast the tibble to a data.frame and then pull a single column
print("Indexing a column from a data.frame becomes a vector")
str(data.frame(excel_sheets_list[[1]])[,1])


# Assign our first sheet to it's own variable
excel_sheets_list_microbes <- ...(excel_sheets_list[[1]])

str(excel_sheets_list_microbes)


# Re-import our microbe.csv file from the data folder if you need to
# microbes <- read_csv(file = "data/microbes.csv", col_names = TRUE, col_types = cols())

# Use default head() parameters
head(microbes)

# Pull just the first 3 rows
head(microbes, ...)


# Let's pull up the last 10 rows to look at!
...(microbes, 10)


# Recall: Use the $ sign to access named columns within your data.frame!

microbes...


# Retrieve a list of unique genera from our data set
...(microbes$genus)


# Two ways to see how many unique entries we have
# ?length
...(unique(microbes$genus))

# or

str(unique(microbes$genus))


# View(microbes)
# Only works in RStudio

# Let's compare str() to glimpse()
str(microbes)


# glimpse gives us less information overall but is also less redundant
...(microbes)


# Set up some vectors for a data.frame
brand <- c("wildrose", "guinness", "grasshoper")
wheat.type <- c("Hard Red Spring", "Hard Red Winter", "Soft Red Winter" )
rating <- c(5, 7, ...)

# Put it all together with cbind (you can use this for data.frames too!)
NA.example <- data.frame(brand, wheat.type, rating)

# Look at our matrix/array
NA.example


# Use the mean() function and see what happens with NA values
mean(rating) # some functions need to be explicitly told what to do with NAs. No errors though!

mean(rating, ...) #Avoid using just "T" as an abbreviation for "TRUE"


counts <- data.frame(Site1 = c(geneA = 2, geneB = 4, geneC = 12, geneD = 8),
                     Site2 = c(geneA = 15, geneB = NA, geneC = 27, geneD = 28),
                     Site3 = c(geneA = 10, geneB = 7, geneC = 13, geneD = NA))

counts

# Notice that we can only pass the function name "mean" and not any parameters
apply(counts, MARGIN = 1, ...)


# Pass parameters in our call
apply(counts, MARGIN = 1, mean, ...)

# Equivalent code - perhaps clearer but more verbose
apply(counts, MARGIN = 1, FUN = function(x) mean(x, na.rm=TRUE))


# Let's check out this vector that contains NA values
na_vector <- c(5, 6, NA, 7, 7, NA)

# This works on vectors...
...(na_vector)

# and data.frames too!
...(counts)


# Let's look at our microbe data for na values
is.na(microbes)


# Before we dig too deep, can we check if there are ANY NA values in our data.frame?
...(is.na(microbes)) # logical (TRUE or FALSE).


# Take a look at na_vector before you start manipulating it
na_vector 

# wrap which() around our is.na() call
...(is.na(na_vector))

# indices where NAs are present in na_vector
na_values <- which(is.na(na_vector)) 

# cut out the na_values indices
removed_na_vector_1 <- na_vector[...] ; removed_na_vector_1


#equivalent to 
removed_na_vector_2 <- na_vector[...(na_vector)] ; removed_na_vector_2


# Which values in microbes are NA? Recall we have 6656 rows of data!
which(is.na(microbes))


# ?complete.cases

# Outputs a logical vector specifying which observations/rows have no missing values across the entire sequence.
head(..., 20)

# Use it wisely to keep complete rows. Pop quiz [x,y] will it be x or y?
head(microbes[...], 5)


# OR use the "!" to flip our boolean result and retrieve all of the incomplete cases!
microbes_NArows <- microbes[...complete.cases(microbes),]

microbes_NArows


# Find the NA values and just replace them with an equivalent value that makes sense to your analysis
microbes_NArows[...,]... <- 0

microbes_NArows


# Find out more about the match operator by using double quotes
# ?"%in%"

# What does %in% return?
str(microbes$genus ... c("Smithella", "Methanobacterium"))


# You can filter your data using basic R commands
# Use the conditional result to index our data.frame
head(...[microbes$genus %in% c("Smithella", "Methanobacterium"),])

# how many rows (entries) do we find with our query?
nrow(microbes[microbes$genus %in% c("Smithella", "Methanobacterium"),])


# A near-equivalent command using the logical OR
# This, however, is a cautionary example about filtering your data. Watch out for this command!

nrow(microbes[(microbes$genus == "Smithella" ... microbes$genus == "Methanobacterium"),])

#The above command will also return any entries with NA in your filtered criteria.
microbes[which(is.na(microbes$genus)),]


# But the syntax using filter is much more human readable
...(microbes, genus == "Smithella" & 
                 genus == "Methanobacterium")


# Filter microbes using the proper logical operator
nrow(filter(microbes, genus == "Smithella" ... 
                      genus == "Methanobacterium"))


#Will this work?
nrow(filter(microbes, genus ... c("Smithella", "Methanobacterium")))


c(1,2,3) + c(10,11)


c(1,2,3,4) + c(10,11)


# Use the right operator to get the job done when filtering with vectors
nrow(filter(microbes, genus ... c("Smithella", "Methanobacterium")))


# Query for samples of either genus Smithella or class Methanobacteria 

head(filter(microbes, 
            genus == "Smithella" | ... == "Methanobacteria"))


# == means "is exactly". 
# Query for rows with abundance equal to 0 from only first replicates
head(filter(microbes, abundance == 0 ... replicate == 1))

# The equivalent call is
head(filter(microbes, 
            abundance == 0... replicate == 1)) # Under the hood this is combined with logical &


# != means "is not"
# Query for microbes with abundance not equal to 0 from replicate 1 data
head(filter(microbes,
           abundance ... 0 & replicate == 1))


# >= means "greater or equal than"
# Query for microbes where abundance is greater than or equal to 0 from replicate 1 data (see above)
head(filter(microbes, 
            abundance >= 0 & replicate == 1))


# >= means "lesser or equal than"
# Query for microbes where abundance is less than or equal to 0 from replicate 1 data
head(filter(microbes, 
            abundance <= 0 & replicate == 1))


# > means "greater than"
# Query for microbes where abundance is above 0 from replicate 1 data
head(filter(microbes, 
            abundance > 0 & replicate == 1))


# Query microbes for those in the proper genus in the first replicate
head(filter(microbes, 
            genus %in% c("Smithella", "Methanobacterium") & replicate == 1))


# >= means "lesser than"
# Query microbes for any instances of abundance <0 in the first replicate.
head(filter(microbes, abundance ... 0 & replicate == 1))


# More about regex
?regex()

# search for matches to argument pattern
?grep()
?grepl()
?regexpr()
?gregexpr()
?regexec()

# perform replacement of the first and all matches respectively.
?sub()
?gsub()


# We just want to know abundance, compound, and genus

head(select(...))


# Select for columns starting with the word "com"
head(select(microbes, abundance, starts_with(...), genus))


# Select a variable using the last three letters of its name using `ends_with()`.
head(select(microbes, ...("und")))


# Explicitly take genus and then add any variables that have a "u" in them
head(select(microbes, genus, ...("u")))

head(select(microbes, genus, ...("u")))


# Sort microbes by descending value of abundance
descending_abundance <- arrange(microbes, ...)
head(descending_abundance)


# Use descending abundance and filter for our criteria
filtered_toluene_descending_abundance <- filter(descending_abundance,
                                                abundance > 10 &
                                                compound == ...) # Extra var 1

# Select just for the genus column
select_genera <- select(..., genus) # Extra var 2
 
# How many of those are unique?
unique_genera <- unique(...) # Extra var 3

# Print the result
unique_genera


# Remember the R evaluates () from the inner to outer
head(filter(microbes, genus == "Smithella" | genus == "Methanobacterium"))

#equivalent to
microbes ... filter(genus == "Smithella" | genus == "Methanobacterium") ... head()
# It really reduces nested parentheses!

#equivalent to
microbes %>% 
    # note there's a period in the first position.
    filter(..., genus == "Smithella" | genus == "Methanobacterium") %>% head()


# Work our data based on what we want
microbes %>% arrange(desc(abundance)) %>% filter(abundance > 10 & compound == "toluene") %>% select(genus) %>% unique()


# Pass our data.frame 
microbes %>% 

    # Arrange by abundance
    arrange(desc(abundance)) %>% 

    # Filter for abundance and compound
    filter(abundance > 10 & compound == "toluene") %>% 

    # Retain just the genus column
    select(genus) %>% 

    # retrieve unique values
    unique()


# Summarise abundance for mean and standard deviation of all rows combined
summarise(...)


# Summarise for mean and sd but make sure we ignore NA values
summarise(microbes, 
          abundance_mean = mean(abundance, ... = TRUE),
          abundance_sd = sd(abundance, ... = TRUE))


# Pass along microbes
microbes %>% 

# group by compound
... %>% 

# Look at the first 10 rows
head(., 10)


# The above code is equivalent to
# Pass along microbes
microbes %>% 

# group by compound
group_by(., compound) %>% 

# Summarise our grouped data
...(abundance_mean = mean(abundance, na.rm = TRUE), 
          abundance_sd = sd(abundance, na.rm = TRUE))


# Here's the equivalent code without piping
summarise(group_by(microbes, compound),
          abundance_mean = mean(abundance, na.rm=TRUE),
          abundance_sd = sd(abundance, na.rm=TRUE))


# Let's just re-initialize microbes here
microbes <- read_csv(file = "data/microbes.csv",
                     col_names = TRUE, col_types = cols())


# Start with our data.frame
microbes %>% 

# Use the mutate command to combine 
mutate(compound_salinity = ...(compound, salinity, sep = "_"), # compound with salinity 
       family_genus = paste(family, genus, sep = "_"))  %>%      # and family with genus

# Peek at our result
head()


# adding columns can also be done using "base R" code:
# This will permanently change microbes
microbes... = paste(microbes$compound, 
                                   microbes$salinity, 
                                   sep = "_")

head(microbes)


# Check the column names before and after removing `compound_salinity`
colnames(microbes)

microbes <- select(microbes, ...) # remove column compound_saline

head(microbes)


microbes %>% 

# Transmute some new columns
...(compound_salinity = paste(compound, salinity, sep ="_"),
          family_genus = paste(family, genus, sep="_"))  %>% 

# Take a peek
head()


# Pass along the dataframe
microbes %>% 
# Make a new column by combining salinity, compound, and group
mutate(... = paste(salinity, compound, group, sep = "_")) %>% 
# Group based on the genus and that new column, sample
group_by(genus, sample) %>% 
# Get a summary of the abundance
summarise(mean = mean(abundance)) %>% 
# Sort by the mean in descending order
arrange(desc(mean)) %>% 
# Take a peek
head()


# Pass along the dataframe
microbes %>% 
# Filter for only Pseudomonas
filter(... == "Pseudomonadaceae")  %>% 
# Make a new column by combining salinity, compound, and group
mutate(sample = paste(salinity, compound, group, sep = "_")) %>% 
# Group based on the genus and that new column, sample
group_by(family, sample) %>% 
# Get a summary of the abundance
summarise(mean = mean(abundance))


getwd()

# Write our data to file
...(x = microbes,
          file = "data/microbes2.csv",
          col_names=TRUE)


write_result <-
# Pass along the dataframe
microbes %>% 
# Filter for only Pseudomonas
filter(family == "Pseudomonadaceae")  %>% 
# Make a new column by combining salinity, compound, and group
mutate(sample = paste(salinity, compound, group, sep = "_")) %>% 
# Group based on the genus and that new column, sample
group_by(family, sample) %>% 
# Get a summary of the abundance
summarise(mean = mean(abundance)) %>% 
# write your file to output
write_csv(x = ., file="data/microbe_summary.csv", col_names=TRUE)

# Take a look at the result of the pipeline
write_result


# install.packages("writexl", dependencies = TRUE)
# library(writexl)

# Write a list to a single xlsx file
...(x = list("microbes_1" = microbes, "microbes_2" = microbes),
           path = "data/microbes.xlsx",
           col_names = TRUE
          )


library(tidyverse)

# Remember the head() function? We'll import our file but just look at the first 6 rows of it
head(read.csv("data/microbes.csv"))


# Note that unlike read_csv() the result here is strictly a dataframe
str(read.csv("data/microbes.csv"))


# Set up our data structures again
na_vector <- c(5, 6, NA, 7, 7, NA)

na_vector

# A data.frame with NA values
counts <- data.frame(Site1 = c(geneA = 2, geneB = 4, geneC = 12, geneD = 8),
                     Site2 = c(geneA = 15, geneB = NA, geneC = 27, geneD = 28),
                     Site3 = c(geneA = 10, geneB = 7, geneC = 13, geneD = NA))

counts


# equivalentish to our previous code our more complex code using is.na() and which() in combination
na.omit(na_vector)


# But under the hood it is doing something slightly different
# see how it works on data.frames?
na.omit(counts)


# Apply the log function to non-NA observations. In this case na.omit can be useful. 
#?na.omit
apply(counts, MARGIN = 1, na.omit(log))

# Read more about apply() to learn more about why our data.frame is now transposed


na_vector <- c(5, 6, NA, 7, 7, NA)
nan_vector <- c(5, 6, NaN, 7, 7, 0/0)

is.na(na_vector)
is.na(nan_vector)

is.na(nan_vector)
is.nan(nan_vector) 

# These type of operations are very useful when working with conditional statements (if else, while, etc.).

Logical operator	Meaning	Example	Result
`==`	equal to	"this" == "that"	FALSE
`!=`	not equal to	4 != 5	TRUE
`>`	greater than	4 > 5	FALSE
`>=`	greater than or equal to	4 >= 5	FALSE
`<`	less than	4 < 5	TRUE
`<=`	less than or equal to	4 <= 5	TRUE

Operator	Description	Use or Result
!	Logical NOT	Converts boolean results into their opposite
&	Element-wise logical AND	Perform element-wise AND result having length of the longer operand
&&	Logical AND	Examines only the first element of the operands resulting into a single length logical vector
\|	Element-wise logical AND	Perform element-wise AND result having length of the longer operand
\|\|	Logical OR	Examines only the first element of the operands resulting into a single length logical vector

Lecture 02: Data wrangling in R¶

0.1.0 About Introduction to R¶

0.1.1 Where is this course headed?¶

0.1.2 How do we get there? Step-by-step.¶

0.2.0 Class Objectives¶

0.3.0 A legend for text format in Jupyter markdown¶

0.4.0 Lecture and data files used in this course¶

0.4.1 Weekly Lecture and skeleton files¶

0.4.2 Live-coding HTML page¶

0.4.3 Post-lecture PDFs and Recordings¶

0.4.4 Dataset 1: /data/miscellaneous.xlsx¶

0.4.5 Dataset 2: /data/microbes.csv¶

1.0.0 Installing and importing packages¶

1.1.0 Locating packages¶

1.2.0 Installing packages for your Jupyter Notebook (on JupyterHub)¶

1.2.1 Will it or won't it install? Check for dependencies!¶

1.2.2 Use library() to load your packages after installation¶

1.3.0 Loading packages from bioconductor requires BiocManager()¶

1.4.0 Skip loading a library with package::function()¶

1.5.0 Packages used in this lesson¶

1.5.1 Quick info for how to install R packages in Anaconda¶

2.0.0 Reading files in R¶

2.1.0 Import data to a tibble with read_csv()¶

2.2.0 Read excel spreadsheets with readxl package¶

2.2.1 Retrieve excel sheet names with excel_sheets()¶

2.2.2 Subset sheet and range within read_excel()¶

2.3.0 lapply() is the list version of apply()¶

2.3.1 The finer details of lapply()¶

2.3.2 A tibble is essentially a data.frame¶

3.0.0 Inspecting your data¶

3.1.0 Helpful commands for inspecting your data¶

3.1.1 Use head() to view the first portion of your data¶

3.1.2 Use tail() to view the latter portion of your data¶

3.1.3 Use unique() to retrieve a list of the unique elements within an object¶

3.1.4 Use length() or str() to retrieve the size of some objects¶

3.1.5 glimpse() and View() show us our data¶

3.2.0 Special data: NA and NaN values¶

3.2.1 Some functions can be told to ignore or remove NA values¶

3.2.2 What happens when we try to use functions via apply() on data with NAs?¶

3.2.3 Use the is.na() function to check your data¶

3.2.4 The any() function evaluates logical vectors¶

3.2.5 Find what you're looking for with the which() function¶

3.2.6 Use complete.cases() to query larger objects¶

3.2.7 Consider just replacing the NAs with something useful¶

4.0.0 A quick introduction to the dplyr (DEE ply er) package¶

4.1.0 Use conditionals to specify subsets of your data based on criteria¶

4.1.1 Use the %in% syntax to compare sets¶

4.2.0 Use the filter() function to replicate %in% and more!¶

4.2.1 Slicing and filtering your data requires the proper use of logical operators¶

4.2.2 A reminder/warning about vectors and recycling¶

4.2.3 When filtering against a long list of criteria, use %in% instead of ==¶

4.2.4 Use filter() to identify matching candidates with criteria across multiple variables¶

4.2.5 Regular expressions can also be used to filter through your data¶

4.3.0 Use select() to subset and order columns in your data frame¶

4.3.1 Use starts_with() and ends_with() helper functions to specify elements from a vector¶

4.3.2 Bonus challenge¶

4.4.0 Sort your data with arrange()¶

4.5.0 Pass along function output to new functions using the piping symbol %>%¶

4.5.1 Using . with %>% denotes the object produced by the last called function¶

4.5.2 Use spacing and new lines to keep track of your directing¶

4.6.0 Retrieve quick summaries of your data with summarise()¶

4.6.1 Don't forget what we've learned about NAs¶

4.7.0 Use group_by() to reorder data based on variable categories¶

4.8.0 Use mutate() to create new columns in your data frame¶

4.8.0.1 Piping will not automatically save your output!¶

4.8.1 Use select() to remove columns¶

4.9.0 Use transmute() to create a new data.frame¶

4.10.0 Challenge¶

4.10.1 Challenge follow-up¶

5.0.0 Writing data files¶

5.1.0 Save your data frame to file with write_csv()¶

5.1.1 Use %>% to direct your output to write_csv()¶

5.2.0 Save your data frame to an excel file with write_xlsx()¶

6.0.0 Class summary¶

6.1.0 Post-lecture assessment (12% of final grade)¶

6.2.0 Acknowledgements¶

6.3.0 Your DataCamp academic subscription¶

6.4.0 Resources¶

7.0.0 Appendix I: Reading files in R using the base package¶

7.1.0 tsv/csv files can be read by read.csv()¶

1.2.2 Use `library()` to load your packages after installation¶

1.3.0 Loading packages from bioconductor requires `BiocManager()`¶

1.4.0 Skip loading a library with `package::function()`¶

2.1.0 Import data to a `tibble` with `read_csv()`¶

2.2.0 Read excel spreadsheets with `readxl` package¶

2.2.1 Retrieve excel sheet names with `excel_sheets()`¶

2.2.2 Subset sheet and range within `read_excel()`¶

2.3.0 `lapply()` is the list version of `apply()`¶

2.3.1 The finer details of `lapply()`¶

2.3.2 A `tibble` is essentially a `data.frame`¶

3.1.1 Use `head()` to view the first portion of your data¶

3.1.2 Use `tail()` to view the latter portion of your data¶

3.1.3 Use `unique()` to retrieve a list of the unique elements within an object¶

3.1.4 Use `length()` or `str()` to retrieve the size of some objects¶

3.1.5 `glimpse()` and `View()` show us our data¶

3.2.0 Special data: `NA` and `NaN` values¶

3.2.1 Some functions can be told to ignore or remove `NA` values¶

3.2.2 What happens when we try to use functions via `apply()` on data with NAs?¶

3.2.3 Use the `is.na()` function to check your data¶

3.2.4 The `any()` function evaluates logical vectors¶

3.2.5 Find what you're looking for with the `which()` function¶

3.2.6 Use `complete.cases()` to query larger objects¶

3.2.7 Consider just replacing the `NA`s with something useful¶

4.0.0 A quick introduction to the `dplyr` (DEE ply er) package¶

4.1.1 Use the `%in%` syntax to compare sets¶

4.2.0 Use the `filter()` function to replicate `%in%` and more!¶

4.2.3 When filtering against a long list of criteria, use `%in%` instead of `==`¶

4.2.4 Use `filter()` to identify matching candidates with criteria across multiple variables¶

4.3.0 Use `select()` to subset and order columns in your data frame¶

4.3.1 Use `starts_with()` and `ends_with()` helper functions to specify elements from a vector¶

4.4.0 Sort your data with `arrange()`¶

4.5.0 Pass along function output to new functions using the piping symbol `%>%`¶

4.5.1 Using `.` with `%>%` denotes the object produced by the last called function¶

4.6.0 Retrieve quick summaries of your data with `summarise()`¶

4.7.0 Use `group_by()` to reorder data based on variable categories¶

4.8.0 Use `mutate()` to create new columns in your data frame¶

4.8.1 Use `select()` to remove columns¶

4.9.0 Use `transmute()` to create a new data.frame¶

5.1.0 Save your data frame to file with `write_csv()`¶

5.1.1 Use `%>%` to direct your output to `write_csv()`¶

5.2.0 Save your data frame to an excel file with `write_xlsx()`¶

7.1.0 tsv/csv files can be read by `read.csv()`¶

8.0.0 Appendix II: Working with `NA` values¶

8.1.0 The `na.omit()` function will remove `NA` entries¶